csv = require 'csv'
T = require './lib/tools.coffee'
$ = require 'jquery'
iconv = require 'iconv-lite'
fs = require 'fs'
url = require 'url'

Spider = T.Spider 
log = T.log
logEnd = T.logEnd
path = './data/'

domain = 'jd.com'
spider = new Spider domain
filename = path+'cat.csv'
fileError = path + 'error.log'
fileRedirect = path + 'redirect.log'
fileKeyword = path+'keyword.csv'
cat = fs.readFileSync(filename).toString()
catArr = cat.split('\n')
fs.unlinkSync fileError if fs.existsSync fileError
fs.unlinkSync fileRedirect if fs.existsSync fileError
fs.unlinkSync fileKeyword if fs.existsSync fileError
spider.on 'onFetchComplete' ,(queueItem, responseBuffer, response)->
    # console.log queueItem
    html = iconv.decode responseBuffer,'GBK'
    dom = $(html)
    nav = dom.find('div.breadcrumb').find('a')
    strArr = []
    str = ''
    for a in nav
        strArr.push ($(a).text())
        strArr.push ($(a).attr('href'))
    str = strArr.join(',')
    # ddArr = []
    for dl in dom.find('#select>dl')
        tdText = $(dl).find('dt').text()
        for dda in $(dl).find('a')
            aurl = queueItem.protocol+'://'+queueItem.host+'/'+$(dda).attr('href')
            # console.log aurl
            ddaStr = [tdText,$(dda).text(),aurl].join(',')
            s = [str,ddaStr].join(',')
            fs.appendFileSync fileKeyword , s + '\n'
    # console.log html
    # console.log html
spider.on 'onFetchheaders' ,(queueItem , responseObject)->
    # console.log responseObject
spider.on 'onFetchstart' ,(queueItem , requestOptions)->

spider.on 'onFetchredirect' ,(queueItem, parsedURL, response )->
    if parsedURL.path.indexOf 'error2.aspx' != -1 
        fs.appendFileSync fileError , JSON.stringify([queueItem,parsedURL]) + '\n'
    else
        fs.appendFileSync fileRedirect , JSON.stringify([queueItem,parsedURL]) + '\n'

spider.on 'onFetchtimeout',(fetchtimeout)->
    fs.appendFileSync fileError,JSON.stringify fetchtimeout
 # spider.on 'onComplete' ,()->
    # spider.crawl()

for i in catArr
    if i isnt '' then [n,u]=i.split(',')
    if u isnt '' then spider.addQueue u   
    # console.log n,u
    # console.log n,u
spider.crawl()
# console.log catArr
# console.log catArr
# // Out